<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - core.h</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_DNn_CORE_H_
<font color='#0000FF'>#define</font> DLIB_DNn_CORE_H_

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='core_abstract.h.html'>core_abstract.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tensor.h.html'>tensor.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>iterator<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>memory<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>type_traits<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../statistics.h.html'>../statistics.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../rand.h.html'>../rand.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../algs.h.html'>../algs.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>utility<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>tuple<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>cmath<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>vector<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tensor_tools.h.html'>tensor_tools.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>type_traits<font color='#5555FF'>&gt;</font>



<font color='#0000FF'>namespace</font> dlib
<b>{</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> int_<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>T::get_learning_rate_multiplier<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type <font color='#5555FF'>=</font> <font color='#979000'>0</font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> obj,
            special_
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> obj.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> , general_<font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#979000'>1</font>; <b>}</b>
    <b>}</b>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> obj<font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> impl::<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font>obj, <font color='#BB00BB'>special_</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> int_<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>T::get_weight_decay_multiplier<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type <font color='#5555FF'>=</font> <font color='#979000'>0</font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> obj,
            special_
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> obj.<font color='#BB00BB'>get_weight_decay_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font> <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> , general_<font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#979000'>1</font>; <b>}</b>
    <b>}</b>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> obj<font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> impl::<font color='#BB00BB'>get_weight_decay_multiplier</font><font face='Lucida Console'>(</font>obj, <font color='#BB00BB'>special_</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>class</font> <b><a name='repeat_input_layer'></a>repeat_input_layer</b> 
        <b>{</b>
            <font color='#009900'>/*!
                None of the declarations in this object are really used. The only reason it
                exists is to allow the repeat object to use a special input layer in its
                internal networks which will cause add_tag_layer objects that happen to be
                right at the input to not create copies of their input tensors.  So
                introducing the repeat_input_layer object allows us to optimize the
                implementation of add_tag_layer for a special case that arises when it's
                used in the context of the repeat layer.
            !*/</font>
        <font color='#0000FF'>public</font>:
            <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>int</u></font> input_type;

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
                forward_iterator ,
                forward_iterator ,
                resizable_tensor<font color='#5555FF'>&amp;</font> 
            <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>,"<font color='#CC0000'>This function should never be called</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat_input_layer<font color='#5555FF'>&amp;</font>, std::ostream<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font><b>{</b><b>}</b>
            <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>repeat_input_layer<font color='#5555FF'>&amp;</font>, std::istream<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font><b>{</b><b>}</b>
            <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> repeat_input_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> out; <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>inline</font> std::string <b><a name='tensor_to_str'></a>tensor_to_str</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> t,
            <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length 
        <font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>t.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> "<font color='#CC0000'></font>";

            std::ostringstream sout;
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>output size=(num:</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>  t.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>";
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>k:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>";
            <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>tellp</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>28</font><font face='Lucida Console'>)</font> sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>";
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nr:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>";
            <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>tellp</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>28</font><font color='#5555FF'>+</font><font color='#979000'>8</font><font face='Lucida Console'>)</font> sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>";
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nc:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>while</font> <font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>tellp</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> min_length<font face='Lucida Console'>)</font> sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>";
            min_length <font color='#5555FF'>=</font> sout.<font color='#BB00BB'>tellp</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\t</font>";
            <font color='#0000FF'>return</font> sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#009900'>// Tell us if T is one of the special layer types (i.e. add_layer, repeat, add_tag_layer, or
</font>    <font color='#009900'>// add_skip_layer).
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font> <font color='#0000FF'>struct</font> <b><a name='is_nonloss_layer_type'></a>is_nonloss_layer_type</b> : std::false_type <b>{</b><b>}</b>;
    <font color='#009900'>// Tell us if T is an instance of add_loss_layer.
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font> <font color='#0000FF'>struct</font> <b><a name='is_loss_layer_type'></a>is_loss_layer_type</b> : std::false_type <b>{</b><b>}</b>;
    <font color='#009900'>// Tell us if T is an instance of add_layer
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font> <font color='#0000FF'>struct</font> <b><a name='is_add_layer'></a>is_add_layer</b> : std::false_type <b>{</b><b>}</b>;

    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font>... n<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='ct_integers_list'></a>ct_integers_list</b> <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> m<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>struct</font> <b><a name='push_back'></a>push_back</b>
            <b>{</b>
                <font color='#0000FF'>typedef</font> ct_integers_list<font color='#5555FF'>&lt;</font>n..., m<font color='#5555FF'>&gt;</font> type;
            <b>}</b>;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> max<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='ct_make_integer_range'></a>ct_make_integer_range</b>
        <b>{</b>
            <font color='#009900'>// recursively call push_back on ct_integers_list to build a range from 1 to max
</font>            <font color='#009900'>// inclusive.
</font>            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> ct_make_integer_range<font color='#5555FF'>&lt;</font>max<font color='#5555FF'>-</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font>::type::<font color='#0000FF'>template</font> push_back<font color='#5555FF'>&lt;</font>max<font color='#5555FF'>&gt;</font>::type type;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='ct_make_integer_range'></a>ct_make_integer_range</b><font color='#5555FF'>&lt;</font><font color='#979000'>0</font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>typedef</font> ct_integers_list<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font> type;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font>... indices, <font color='#0000FF'>typename</font> Tuple<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='tuple_subset'></a>tuple_subset</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> Tuple<font color='#5555FF'>&amp;</font> item, 
            ct_integers_list<font color='#5555FF'>&lt;</font>indices...<font color='#5555FF'>&gt;</font>
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>make_tuple</font><font face='Lucida Console'>(</font>std::get<font color='#5555FF'>&lt;</font>indices<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> std::<font color='#BB00BB'>make_tuple</font><font face='Lucida Console'>(</font>std::get<font color='#5555FF'>&lt;</font>indices<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> Head, <font color='#0000FF'>typename</font>... Tail<font color='#5555FF'>&gt;</font>
        std::tuple<font color='#5555FF'>&lt;</font>Tail...<font color='#5555FF'>&gt;</font> <b><a name='basic_tuple_tail'></a>basic_tuple_tail</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>Head, Tail...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>tuple_subset</font><font face='Lucida Console'>(</font>item, <font color='#0000FF'>typename</font> ct_make_integer_range<font color='#5555FF'>&lt;</font><font color='#0000FF'>sizeof</font>...<font face='Lucida Console'>(</font>Tail<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>type</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        std::tuple<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font> <b><a name='tuple_flatten'></a>tuple_flatten</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> t<font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#0000FF'>return</font> std::<font color='#BB00BB'>make_tuple</font><font face='Lucida Console'>(</font>t<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font>... T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='tuple_flatten'></a>tuple_flatten</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font><font color='#BB00BB'>tuple_flatten</font><font face='Lucida Console'>(</font>item, <font color='#0000FF'>typename</font> ct_make_integer_range<font color='#5555FF'>&lt;</font><font color='#0000FF'>sizeof</font>...<font face='Lucida Console'>(</font>T<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>type</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>tuple_flatten</font><font face='Lucida Console'>(</font>item, <font color='#0000FF'>typename</font> ct_make_integer_range<font color='#5555FF'>&lt;</font><font color='#0000FF'>sizeof</font>...<font face='Lucida Console'>(</font>T<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>type</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font>... indices, <font color='#0000FF'>typename</font>... T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='tuple_flatten'></a>tuple_flatten</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item, 
            ct_integers_list<font color='#5555FF'>&lt;</font>indices...<font color='#5555FF'>&gt;</font>
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>tuple_cat</font><font face='Lucida Console'>(</font><font color='#BB00BB'>tuple_flatten</font><font face='Lucida Console'>(</font>std::get<font color='#5555FF'>&lt;</font>indices<font color='#5555FF'>-</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> std::<font color='#BB00BB'>tuple_cat</font><font face='Lucida Console'>(</font><font color='#BB00BB'>tuple_flatten</font><font face='Lucida Console'>(</font>std::get<font color='#5555FF'>&lt;</font>indices<font color='#5555FF'>-</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='tuple_head_helper'></a>tuple_head_helper</b>
        <b>{</b>
            <font color='#0000FF'>typedef</font> T type;
            <font color='#0000FF'>static</font> <font color='#0000FF'>const</font> type<font color='#5555FF'>&amp;</font> <b><a name='get'></a>get</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> 
            <b>{</b>
                <font color='#0000FF'>return</font> item;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font>... U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='tuple_head_helper'></a>tuple_head_helper</b><font color='#5555FF'>&lt;</font>std::tuple<font color='#5555FF'>&lt;</font>T, U...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> tuple_head_helper<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::type type;
            <font color='#0000FF'>static</font> <font color='#0000FF'>const</font> type<font color='#5555FF'>&amp;</font> <b><a name='get'></a>get</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>T,U...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> 
            <b>{</b>
                <font color='#0000FF'>return</font> tuple_head_helper<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>get</font><font face='Lucida Console'>(</font>std::get<font color='#5555FF'>&lt;</font><font color='#979000'>0</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font> <font color='#0000FF'>struct</font> <b><a name='alwaysbool'></a>alwaysbool</b> <b>{</b> <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>bool</u></font> type; <b>}</b>;
        <font color='#009900'>// one more structure for VS 2015 UP3 support workaround
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font> <font color='#0000FF'>struct</font> <b><a name='alwaysbool2'></a>alwaysbool2</b> <b>{</b> <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>bool</u></font> type; <b>}</b>;

        resizable_tensor<font color='#5555FF'>&amp;</font> <b><a name='rt'></a>rt</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// The significance of a layer's backward method requiring forward's outputs is
</font>        <font color='#009900'>// that such as layer can't have an in-place layer stacked on top of it because
</font>        <font color='#009900'>// in-place layers overwrite the output of the layer they sit on top of.
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='backward_requires_forward_output'></a>backward_requires_forward_output</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='backward_requires_forward_output'></a>backward_requires_forward_output</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='backward_requires_forward_output'></a>backward_requires_forward_output</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='backward_requires_forward_output'></a>backward_requires_forward_output</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='has_inplace_backward'></a>has_inplace_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool2<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='has_inplace_backward'></a>has_inplace_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool2<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='has_inplace_backward'></a>has_inplace_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool2<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='has_inplace_backward'></a>has_inplace_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool2<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='is_inplace_layer'></a>is_inplace_layer</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub 
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool2<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>sub,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        constexpr <font color='#0000FF'>auto</font> <b><a name='is_inplace_layer'></a>is_inplace_layer</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>typename</font> alwaysbool<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>forward_inplace</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_backward'></a>call_layer_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font>computed_output,gradient_input,sub,params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font>computed_output,gradient_input,sub,params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_backward'></a>call_layer_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> , 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font>gradient_input,sub,params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            layer.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font>gradient_input,sub,params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_backward'></a>call_layer_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font>computed_output,gradient_input,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font>computed_output,gradient_input,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_backward'></a>call_layer_backward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> , 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font>gradient_input,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            layer.<font color='#BB00BB'>backward_inplace</font><font face='Lucida Console'>(</font>gradient_input,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_forward'></a>call_layer_forward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*data_output*/</font>
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>sub,<font color='#BB00BB'>rt</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// This overload of call_layer_forward() is here because this template
</font>            <font color='#009900'>// naturally gets instantiated but only on code paths that never get executed.
</font>            <font color='#009900'>// So rather than writing a bunch of hard to read template magic around call
</font>            <font color='#009900'>// sites we just have this overload that doesn't do anything (and an assert to
</font>            <font color='#009900'>// make sure that's the case).
</font>            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, "<font color='#CC0000'>This should never happen</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_forward'></a>call_layer_forward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, 
            resizable_tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>sub,data_output<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            layer.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>sub,data_output<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_forward'></a>call_layer_forward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>forward_inplace</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,data_output<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            layer.<font color='#BB00BB'>forward_inplace</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,data_output<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> layer_type, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='call_layer_forward'></a>call_layer_forward</b><font face='Lucida Console'>(</font>
            layer_type<font color='#5555FF'>&amp;</font> layer,
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, 
            resizable_tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>layer.<font color='#BB00BB'>forward_inplace</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,data_output<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>data_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                data_output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            layer.<font color='#BB00BB'>forward_inplace</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>data_output<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>


    <b>}</b> <font color='#009900'>// end namespace impl
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font>... T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>typename</font> impl::tuple_head_helper<font color='#5555FF'>&lt;</font>std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>::type <b><a name='tuple_head'></a>tuple_head</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
    <font face='Lucida Console'>)</font> 
    <b>{</b>
        <font color='#0000FF'>return</font> impl::tuple_head_helper<font color='#5555FF'>&lt;</font>std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>get</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font>... T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>auto</font> <b><a name='tuple_tail'></a>tuple_tail</b><font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
    <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>impl::<font color='#BB00BB'>basic_tuple_tail</font><font face='Lucida Console'>(</font>impl::<font color='#BB00BB'>tuple_flatten</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>return</font> impl::<font color='#BB00BB'>basic_tuple_tail</font><font face='Lucida Console'>(</font>impl::<font color='#BB00BB'>tuple_flatten</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>inline</font> std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font> <b><a name='tuple_tail'></a>tuple_tail</b><font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
    <font face='Lucida Console'>)</font> 
    <b>{</b>
        <font color='#0000FF'>return</font> item;
    <b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='sstack'></a>sstack</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> T value_type;

        <b><a name='sstack'></a>sstack</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;

        <b><a name='sstack'></a>sstack</b> <font face='Lucida Console'>(</font>
            T<font color='#5555FF'>*</font> data_,
            <font color='#0000FF'><u>size_t</u></font> s
        <font face='Lucida Console'>)</font> : data<font face='Lucida Console'>(</font>data_<font face='Lucida Console'>)</font>, mysize<font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'>const</font> T<font color='#5555FF'>&amp;</font> <b><a name='top'></a>top</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't call top() on an empty stack</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>data;
        <b>}</b>
        T<font color='#5555FF'>&amp;</font> <b><a name='top'></a>top</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>  
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't call top() on an empty stack</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>data;
        <b>}</b>

        <font color='#0000FF'><u>size_t</u></font> <b><a name='size'></a>size</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> mysize; <b>}</b>

        sstack <b><a name='pop'></a>pop</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>=</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>You can't pop more things from the stack than it has in it.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>sstack</font><font face='Lucida Console'>(</font>data<font color='#5555FF'>+</font>num, mysize<font color='#5555FF'>-</font>num<font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:

        T<font color='#5555FF'>*</font> data;
        <font color='#0000FF'><u>size_t</u></font> mysize;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    sstack<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font> <b><a name='make_sstack'></a>make_sstack</b><font face='Lucida Console'>(</font>std::vector<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>return</font> sstack<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item.<font color='#BB00BB'>data</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, item.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> dimpl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first <font color='#5555FF'>=</font> <font color='#979000'>true</font>, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>=</font><font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>class</font> <b><a name='subnet_wrapper'></a>subnet_wrapper</b>
        <b>{</b>
            <font color='#009900'>/*!
                WHAT THIS OBJECT REPRESENTS
                    This is a tool that makes an add_layer or add_loss_layer object
                    expose only the part of its interface defined by the SUBNET
                    type in layers_abstract.h.  This way, when we pass subnetwork
                    objects to the layer callbacks those callbacks won't be able to 
                    interact with the subnetworks in a way other than specified 
                    by the SUBNET interface spec.

                    We also allow the top layer of a subnet_wrapper stack to call the
                    private_get_output() and private_get_gradient_input() functions.  This
                    way, layers that have had their output/gradient overwritten by in-place
                    layers can only be accessed from the in-place layers that sit directly
                    on top of them since those in-place layers are the only layers that
                    know how to interact with them properly.
            !*/</font>

        <font color='#0000FF'>public</font>:
            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;
            subnet_wrapper<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;

            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> l_, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> sef<font face='Lucida Console'>)</font> : l<font face='Lucida Console'>(</font>l_<font face='Lucida Console'>)</font>,_sample_expansion_factor<font face='Lucida Console'>(</font>sef<font face='Lucida Console'>)</font> <b>{</b><b>}</b>
            <font color='#009900'>// Not much here because in this case T is one of the input layer types 
</font>            <font color='#009900'>// that doesn't have anything in it.
</font>            <font color='#0000FF'>typedef</font> T layer_details_type;
            <font color='#0000FF'>const</font> layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l; <b>}</b>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _sample_expansion_factor; <b>}</b>
        <font color='#0000FF'>private</font>:
            T<font color='#5555FF'>&amp;</font> l;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> _sample_expansion_factor;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>class</font> <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font color='#5555FF'>&lt;</font>T,<font color='#979000'>true</font>, <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>is_nonloss_layer_type<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>

        <font color='#0000FF'>public</font>:
            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;
            subnet_wrapper<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;

            <font color='#0000FF'>typedef</font> T wrapped_type;
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> T::num_computational_layers;
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> T::num_layers;
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> T::layer_details_type layer_details_type;

            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> l_, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> : l<font face='Lucida Console'>(</font>l_<font face='Lucida Console'>)</font>,subnetwork<font face='Lucida Console'>(</font>l.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, l.sample_expansion_factor<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>
            tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

            <font color='#0000FF'>const</font> layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

            <font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type,<font color='#979000'>false</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
            subnet_wrapper<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type,<font color='#979000'>false</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'>private</font>:
            T<font color='#5555FF'>&amp;</font> l;
            subnet_wrapper<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type,<font color='#979000'>false</font><font color='#5555FF'>&gt;</font> subnetwork;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>class</font> <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font color='#5555FF'>&lt;</font>T,<font color='#979000'>false</font>, <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>is_nonloss_layer_type<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>

        <font color='#0000FF'>public</font>:
            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;
            subnet_wrapper<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;

            <font color='#0000FF'>typedef</font> T wrapped_type;
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> T::num_computational_layers;
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> T::num_layers;
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> T::layer_details_type layer_details_type;

            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> l_, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> : l<font face='Lucida Console'>(</font>l_<font face='Lucida Console'>)</font>,subnetwork<font face='Lucida Console'>(</font>l.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, l.sample_expansion_factor<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>
            tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

            <font color='#0000FF'>const</font> layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

            <font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type,<font color='#979000'>false</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
            subnet_wrapper<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type,<font color='#979000'>false</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> l.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'>private</font>:
            T<font color='#5555FF'>&amp;</font> l;
            subnet_wrapper<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type,<font color='#979000'>false</font><font color='#5555FF'>&gt;</font> subnetwork;
        <b>}</b>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> LAYER_DETAILS, <font color='#0000FF'>typename</font> SUBNET, <font color='#0000FF'>typename</font> enabled <font color='#5555FF'>=</font> <font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> add_layer;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_nonloss_layer_type'></a>is_nonloss_layer_type</b><font color='#5555FF'>&lt;</font>add_layer<font color='#5555FF'>&lt;</font>T,U<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> LAYER_DETAILS, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_layer'></a>add_layer</b><font color='#5555FF'>&lt;</font>LAYER_DETAILS,SUBNET,
            <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>is_nonloss_layer_type<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> LAYER_DETAILS layer_details_type;
        <font color='#0000FF'>typedef</font> SUBNET subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> subnet_type::input_type input_type;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> subnet_type::num_layers <font color='#5555FF'>+</font> <font color='#979000'>1</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> subnet_type::num_computational_layers <font color='#5555FF'>+</font> <font color='#979000'>1</font>;

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>:
            subnetwork<font face='Lucida Console'>(</font><font color='#0000FF'>new</font> subnet_type<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            details <font color='#5555FF'>=</font> item.details;
            subnetwork.<font color='#BB00BB'>reset</font><font face='Lucida Console'>(</font><font color='#0000FF'>new</font> <font color='#BB00BB'>subnet_type</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>item.subnetwork<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            this_layer_setup_called <font color='#5555FF'>=</font> item.this_layer_setup_called;
            gradient_input_is_stale <font color='#5555FF'>=</font> item.gradient_input_is_stale;
            get_output_and_gradient_input_disabled <font color='#5555FF'>=</font> item.get_output_and_gradient_input_disabled;
            x_grad <font color='#5555FF'>=</font> item.x_grad;
            cached_output <font color='#5555FF'>=</font> item.cached_output; 
            params_grad <font color='#5555FF'>=</font> item.params_grad; 
            temp_tensor <font color='#5555FF'>=</font> item.temp_tensor;
        <b>}</b>
        add_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>add_layer</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>.<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;<b>}</b>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <b>}</b>
        add_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='dimpl'></a>dimpl</b>::subnet_wrapper;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_tag_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_skip_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> repeat;

        <font color='#009900'>// Allow copying networks from one to another as long as their corresponding 
</font>        <font color='#009900'>// layers can be constructed from each other.
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> :
            details<font face='Lucida Console'>(</font>item.layer_details<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            subnetwork<font face='Lucida Console'>(</font><font color='#0000FF'>new</font> subnet_type<font face='Lucida Console'>(</font>item.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font>item.this_layer_setup_called<font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font>item.gradient_input_is_stale<font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font>item.get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>,
            x_grad<font face='Lucida Console'>(</font>item.x_grad<font face='Lucida Console'>)</font>,
            cached_output<font face='Lucida Console'>(</font>item.cached_output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> LAYER_DETAILS<font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font>, 
            subnetwork<font face='Lucida Console'>(</font><font color='#0000FF'>new</font> subnet_type<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b> 
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> std::is_constructible<font color='#5555FF'>&lt;</font>LAYER_DETAILS,T<font color='#5555FF'>&gt;</font>::value;
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b><font color='#5555FF'>&lt;</font>std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font>,U...<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> disable_forwarding_constr<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::type...<font color='#5555FF'>&gt;</font>::value;
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b><font color='#5555FF'>&lt;</font>std::tuple<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>,U...<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> disable_forwarding_constr<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>::value;
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b><font color='#5555FF'>&lt;</font>std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>,U...<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b><font color='#5555FF'>&lt;</font>add_layer<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> ...T,
            <font color='#0000FF'>typename</font> <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font color='#5555FF'>!</font>disable_forwarding_constr<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::type...<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type
            <font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            subnetwork<font face='Lucida Console'>(</font><font color='#0000FF'>new</font> subnet_type<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            LAYER_DETAILS<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            subnetwork<font face='Lucida Console'>(</font><font color='#0000FF'>new</font> subnet_type<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T, <font color='#0000FF'>typename</font> LD, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>LD,U...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>tuple_head<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            subnetwork<font face='Lucida Console'>(</font><font color='#0000FF'>new</font> subnet_type<font face='Lucida Console'>(</font>tuple_tail<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font>,std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T, <font color='#0000FF'>typename</font> LD, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>,
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>LD,U...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>layer_det,args...<font face='Lucida Console'>)</font> <b>{</b> <b>}</b>

        <b><a name='add_layer'></a>add_layer</b> <font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>, 
            LAYER_DETAILS<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>layer_det, args...<font face='Lucida Console'>)</font> <b>{</b> <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,data<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>temp_tensor<font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x, <font color='#5555FF'>&amp;</font>x<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>subnetwork<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>this_layer_setup_called<font face='Lucida Console'>)</font>
            <b>{</b>
                details.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>wsub<font face='Lucida Console'>)</font>;
                this_layer_setup_called <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            <b>}</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>details, wsub, <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>details, wsub, cached_output<font face='Lucida Console'>)</font>;

            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font>add_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                <font color='#0000FF'>return</font> <font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font>resizable_tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>cached_output<font face='Lucida Console'>)</font>; 
        <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>this_layer_operates_inplace</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>gradient_input_is_stale<font face='Lucida Console'>)</font>
                <b>{</b>
                    gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
                    x_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    x_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>
                <font color='#0000FF'>return</font> x_grad; 
            <b>}</b>
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='disable_output_and_gradient_getters'></a>disable_output_and_gradient_getters</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> get_output_and_gradient_input_disabled <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <b>}</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> dlib::<font color='#BB00BB'>error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Accessing this layer's get_output() is disabled because an in-place layer has been stacked on top of it.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; 
        <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> dlib::<font color='#BB00BB'>error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Accessing this layer's get_gradient_input() is disabled because an in-place layer has been stacked on top of it.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_final_data_gradient'></a>get_final_data_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>get_final_data_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x, <font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input<font face='Lucida Console'>)</font>
        <b>{</b>
            dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>subnetwork<font face='Lucida Console'>)</font>;
            params_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>details.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            impl::<font color='#BB00BB'>call_layer_backward</font><font face='Lucida Console'>(</font>details, <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                gradient_input, wsub, <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>; 

            <font color='#009900'>// zero out get_gradient_input()
</font>            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b><font face='Lucida Console'>(</font>sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> solvers, <font color='#0000FF'><u>double</u></font> learning_rate<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>solvers.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font>num_computational_layers<font face='Lucida Console'>)</font>;
            <font color='#009900'>// Don't try to adjust the parameters if this layer doesn't have any or the
</font>            <font color='#009900'>// learning rate is disabled for this layer.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>params_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font>details<font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> step <font color='#5555FF'>=</font> solvers.<font color='#BB00BB'>top</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>learning_rate, details, <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>details.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, details.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, step<font face='Lucida Console'>)</font>;
            <b>}</b>
            subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>update_parameters</font><font face='Lucida Console'>(</font>solvers.<font color='#BB00BB'>pop</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, learning_rate<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>subnetwork; <b>}</b>
        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>subnetwork; <b>}</b>

        <font color='#0000FF'>const</font> layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> details; <b>}</b> 
        layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> details; <b>}</b> 

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            x_grad.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cached_output.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            params_grad.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            temp_tensor.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>item.subnetwork, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.details, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.this_layer_setup_called, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gradient_input_is_stale, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.get_output_and_gradient_input_disabled, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.x_grad, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.cached_output, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params_grad, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> version <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> version <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::add_layer.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>item.subnetwork, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.details, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.this_layer_setup_called, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gradient_input_is_stale, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.get_output_and_gradient_input_disabled, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.x_grad, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.cached_output, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params_grad, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> impl::<font color='#BB00BB'>tensor_to_str</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, min_length<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";
            <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, idx<font color='#5555FF'>+</font><font color='#979000'>1</font>, min_length<font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_operates_inplace'></a>this_layer_operates_inplace</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#009900'>// This layer can run in-place if it's an in-place capable layer and also if
</font>            <font color='#009900'>// the layer it's on top of doesn't need its own output tensor (since in-place
</font>            <font color='#009900'>// layers overwrite that tensor)
</font>            <font color='#0000FF'>return</font> impl::<font color='#BB00BB'>is_inplace_layer</font><font face='Lucida Console'>(</font>details, <font color='#5555FF'>*</font>subnetwork<font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#5555FF'>!</font>subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>this_layer_requires_forward_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_requires_forward_output'></a>this_layer_requires_forward_output</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#0000FF'>return</font> impl::<font color='#BB00BB'>backward_requires_forward_output</font><font face='Lucida Console'>(</font>details, <font color='#5555FF'>*</font>subnetwork<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='swap'></a>swap</b><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>subnetwork,item.subnetwork<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>details, item.details<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>this_layer_setup_called, item.this_layer_setup_called<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>gradient_input_is_stale, item.gradient_input_is_stale<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>get_output_and_gradient_input_disabled, item.get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>x_grad, item.x_grad<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>cached_output, item.cached_output<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>params_grad, item.params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>


        LAYER_DETAILS details;
        std::unique_ptr<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> subnetwork;
        <font color='#0000FF'><u>bool</u></font> this_layer_setup_called;
        <font color='#0000FF'><u>bool</u></font> gradient_input_is_stale;
        <font color='#0000FF'><u>bool</u></font> get_output_and_gradient_input_disabled;
        <font color='#009900'>// Note that if this_layer_operates_inplace()==true then x_grad and cached_output
</font>        <font color='#009900'>// are not used at all.  Instead, this layer uses these variables from the lower
</font>        <font color='#009900'>// layer.
</font>        resizable_tensor x_grad;
        resizable_tensor cached_output; 

        resizable_tensor params_grad; 

        <font color='#009900'>// temp_tensor doesn't logically contribute to the state of this object.  
</font>        <font color='#009900'>// It is here only to prevent it from being reallocated over and over.
</font>        resizable_tensor temp_tensor;

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_add_layer'></a>is_add_layer</b><font color='#5555FF'>&lt;</font>add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_add_layer'></a>is_add_layer</b><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_add_layer'></a>is_add_layer</b><font color='#5555FF'>&lt;</font>add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_add_layer'></a>is_add_layer</b><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#009900'>// This version of add_layer handles the special case where the subnetwork being given is
</font><font color='#009900'>// just an input layer object.
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> LAYER_DETAILS, <font color='#0000FF'>typename</font> INPUT_LAYER, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_layer'></a>add_layer</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> LAYER_DETAILS layer_details_type;
        <font color='#0000FF'>typedef</font> INPUT_LAYER subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> INPUT_LAYER::input_type input_type;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>: 
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <b>}</b>
        add_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='dimpl'></a>dimpl</b>::subnet_wrapper;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_tag_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_skip_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> repeat;

        <font color='#009900'>// Allow copying networks from one to another as long as their corresponding 
</font>        <font color='#009900'>// layers can be constructed from each other.
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>:
            input_layer<font face='Lucida Console'>(</font>item.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            details<font face='Lucida Console'>(</font>item.layer_details<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font>item.this_layer_setup_called<font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font>item.gradient_input_is_stale<font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font>item._sample_expansion_factor<font face='Lucida Console'>)</font>,
            x_grad<font face='Lucida Console'>(</font>item.x_grad<font face='Lucida Console'>)</font>,
            cached_output<font face='Lucida Console'>(</font>item.cached_output<font face='Lucida Console'>)</font>,
            grad_final<font face='Lucida Console'>(</font>item.grad_final<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> LAYER_DETAILS<font color='#5555FF'>&amp;</font> layer_det
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font>, 
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> INPUT_LAYER<font color='#5555FF'>&amp;</font> il 
        <font face='Lucida Console'>)</font> : 
            input_layer<font face='Lucida Console'>(</font>il<font face='Lucida Console'>)</font>, 
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            LAYER_DETAILS<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> layer_det
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            LAYER_DETAILS layer_det, 
            INPUT_LAYER il
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            input_layer<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>il<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            this_layer_setup_called<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            get_output_and_gradient_input_disabled<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>,
            <font color='#0000FF'>const</font> LAYER_DETAILS<font color='#5555FF'>&amp;</font> layer_det
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>,
            LAYER_DETAILS<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> layer_det
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>,
            LAYER_DETAILS layer_det, 
            INPUT_LAYER il
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>layer_det,il<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>LAYER_DETAILS<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> layer_det
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>tuple_head<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <b><a name='add_layer'></a>add_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::tuple<font color='#5555FF'>&lt;</font>LAYER_DETAILS<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> layer_det,
            INPUT_LAYER il
        <font face='Lucida Console'>)</font> : add_layer<font face='Lucida Console'>(</font>tuple_head<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font>,il<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            input_layer.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin, iend, data<font face='Lucida Console'>)</font>;
            <font color='#009900'>// make sure the input layer's to_tensor() function is implemented properly.
</font>            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The input layer can't produce fewer output tensors than there are inputs.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>,
            "<font color='#CC0000'>The number of tensors produced by the input layer must be an integer multiple of the number of input objects.</font>"<font face='Lucida Console'>)</font>;

            _sample_expansion_factor <font color='#5555FF'>=</font> data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font>;
            data.<font color='#BB00BB'>async_copy_to_device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>temp_tensor<font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x, <font color='#5555FF'>&amp;</font>x<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='forward'></a>forward</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You must call to_tensor() before this function can be used.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            subnet_wrapper <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>x, grad_final, _sample_expansion_factor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>this_layer_setup_called<font face='Lucida Console'>)</font>
            <b>{</b>
                details.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>wsub<font face='Lucida Console'>)</font>;
                this_layer_setup_called <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            <b>}</b>
            impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>details, wsub, cached_output<font face='Lucida Console'>)</font>;
            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font>resizable_tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>cached_output<font face='Lucida Console'>)</font>; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>gradient_input_is_stale<font face='Lucida Console'>)</font>
            <b>{</b>
                gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
                x_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                x_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> x_grad; 
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='disable_output_and_gradient_getters'></a>disable_output_and_gradient_getters</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> get_output_and_gradient_input_disabled <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <b>}</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> dlib::<font color='#BB00BB'>error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Accessing this layer's get_output() is disabled because an in-place layer has been stacked on top of it.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; 
        <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> dlib::<font color='#BB00BB'>error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Accessing this layer's get_gradient_input() is disabled because an in-place layer has been stacked on top of it.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_final_data_gradient'></a>get_final_data_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> grad_final; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x, <font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// make sure grad_final is initialized to 0
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>x, grad_final<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                grad_final.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            grad_final <font color='#5555FF'>=</font> <font color='#979000'>0</font>;  

            subnet_wrapper <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>x, grad_final, _sample_expansion_factor<font face='Lucida Console'>)</font>;
            params_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>details.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            impl::<font color='#BB00BB'>call_layer_backward</font><font face='Lucida Console'>(</font>details, <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                gradient_input, wsub, <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// zero out get_gradient_input()
</font>            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b><font face='Lucida Console'>(</font>sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> solvers, <font color='#0000FF'><u>double</u></font> learning_rate<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>solvers.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font>num_computational_layers<font face='Lucida Console'>)</font>;
            <font color='#009900'>// Don't try to adjust the parameters if this layer doesn't have any or the
</font>            <font color='#009900'>// learning rate is disabled for this layer.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>params_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font>details<font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> 
            <b>{</b>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> step <font color='#5555FF'>=</font> solvers.<font color='#BB00BB'>top</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>learning_rate, details, <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>params_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>details.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, details.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, step<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>  <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> input_layer; <b>}</b> 
        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> input_layer; <b>}</b> 

        <font color='#0000FF'>const</font> layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> details; <b>}</b> 
        layer_details_type<font color='#5555FF'>&amp;</font> <b><a name='layer_details'></a>layer_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> details; <b>}</b> 

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _sample_expansion_factor; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            x_grad.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            grad_final.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cached_output.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            params_grad.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            temp_tensor.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.input_layer, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.details, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.this_layer_setup_called, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gradient_input_is_stale, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.get_output_and_gradient_input_disabled, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.x_grad, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.cached_output, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.grad_final, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item._sample_expansion_factor, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font><font color='#979000'>2</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> version <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> version <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::add_layer.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.input_layer, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.details, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.this_layer_setup_called, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gradient_input_is_stale, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.get_output_and_gradient_input_disabled, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.x_grad, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.cached_output, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.grad_final, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>3</font><font face='Lucida Console'>)</font>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item._sample_expansion_factor, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                item._sample_expansion_factor <font color='#5555FF'>=</font> <font color='#979000'>1</font>; <font color='#009900'>// all layer types set this to 1 in older dlib versions, so that's what we put here.
</font>        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> impl::<font color='#BB00BB'>tensor_to_str</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, min_length<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";

            <font color='#009900'>// Don't print the repeat_input_layer since it doesn't exist from the user's
</font>            <font color='#009900'>// point of view.  It's just an artifact of how repeat&lt;&gt; works.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>std::is_same<font color='#5555FF'>&lt;</font>subnet_type, impl::repeat_input_layer<font color='#5555FF'>&gt;</font>::value<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx<font color='#5555FF'>+</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_requires_forward_output'></a>this_layer_requires_forward_output</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b>
            subnet_wrapper <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>grad_final, grad_final, _sample_expansion_factor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> impl::<font color='#BB00BB'>backward_requires_forward_output</font><font face='Lucida Console'>(</font>details, wsub<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>class</font> <b><a name='subnet_wrapper'></a>subnet_wrapper</b>
        <b>{</b>
        <font color='#0000FF'>public</font>:
            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x_, resizable_tensor<font color='#5555FF'>&amp;</font> grad_final_, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> sef<font face='Lucida Console'>)</font> :
                x<font face='Lucida Console'>(</font>x_<font face='Lucida Console'>)</font>, grad_final<font face='Lucida Console'>(</font>grad_final_<font face='Lucida Console'>)</font>, _sample_expansion_factor<font face='Lucida Console'>(</font>sef<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <b><a name='subnet_wrapper'></a>subnet_wrapper</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;
            subnet_wrapper<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> subnet_wrapper<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>delete</font>;

            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _sample_expansion_factor;<b>}</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> x; <b>}</b>
            tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
            <b>{</b> 
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>x, grad_final<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    grad_final.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
                    grad_final <font color='#5555FF'>=</font> <font color='#979000'>0</font>;  
                <b>}</b>
                <font color='#0000FF'>return</font> grad_final; 
            <b>}</b>

        <font color='#0000FF'>private</font>:
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x;
            resizable_tensor<font color='#5555FF'>&amp;</font> grad_final;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> _sample_expansion_factor;
        <b>}</b>;

        <font color='#0000FF'><u>void</u></font> <b><a name='swap'></a>swap</b><font face='Lucida Console'>(</font>add_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>input_layer, item.input_layer<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>details, item.details<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>this_layer_setup_called, item.this_layer_setup_called<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>gradient_input_is_stale, item.gradient_input_is_stale<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>get_output_and_gradient_input_disabled, item.get_output_and_gradient_input_disabled<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>x_grad, item.x_grad<font face='Lucida Console'>)</font>; 
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>cached_output, item.cached_output<font face='Lucida Console'>)</font>; 
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>grad_final, item.grad_final<font face='Lucida Console'>)</font>; 
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>_sample_expansion_factor, item._sample_expansion_factor<font face='Lucida Console'>)</font>; 
        <b>}</b>

        subnet_type input_layer;
        LAYER_DETAILS details;
        <font color='#0000FF'><u>bool</u></font> this_layer_setup_called;
        <font color='#0000FF'><u>bool</u></font> gradient_input_is_stale;
        <font color='#0000FF'><u>bool</u></font> get_output_and_gradient_input_disabled;
        <font color='#0000FF'>mutable</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> _sample_expansion_factor;
        resizable_tensor x_grad; 
        resizable_tensor cached_output; 
        resizable_tensor grad_final;

        <font color='#009900'>// The following 2 objects don't logically contribute to the state of this class.
</font>        <font color='#009900'>// They are only here to prevent them from being reallocated over and over in
</font>        <font color='#009900'>// member functions.
</font>        resizable_tensor params_grad; 
        resizable_tensor temp_tensor; 
    <b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> ID, <font color='#0000FF'>typename</font> SUBNET, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>=</font><font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> add_tag_layer;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b><font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='tag_id'></a>tag_id</b>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag<font color='#5555FF'>&lt;</font>impl::repeat_input_layer<font color='#5555FF'>&gt;</font>::id;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> ID, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_tag_layer'></a>add_tag_layer</b><font color='#5555FF'>&lt;</font>ID,SUBNET,
            <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>is_nonloss_layer_type<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> SUBNET subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> subnet_type::input_type input_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>int</u></font> layer_details_type; <font color='#009900'>// not really used anywhere, but required by subnet_wrapper.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> subnet_type::num_layers <font color='#5555FF'>+</font> <font color='#979000'>1</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> subnet_type::num_computational_layers;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> ID;

        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>;
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_tag_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_tag_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&lt;</font>ID,T<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : subnetwork<font face='Lucida Console'>(</font>item.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font>
            T ...args
        <font face='Lucida Console'>)</font> : 
            subnetwork<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,data<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>subnetwork</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>subnetwork</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_final_data_gradient'></a>get_final_data_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>get_final_data_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x,gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b><font face='Lucida Console'>(</font>sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> solvers, <font color='#0000FF'><u>double</u></font> learning_rate<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>update_parameters</font><font face='Lucida Console'>(</font>solvers, learning_rate<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.subnetwork, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::add_tag_layer.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.subnetwork, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> impl::<font color='#BB00BB'>tensor_to_str</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, min_length<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>tag</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> ID <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";
            <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, idx<font color='#5555FF'>+</font><font color='#979000'>1</font>, min_length<font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='dimpl'></a>dimpl</b>::subnet_wrapper;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_tag_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_skip_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> repeat;

        <font color='#009900'>// You wouldn't put a tag on a layer if you didn't want to access its forward
</font>        <font color='#009900'>// outputs.  So this is always true.
</font>        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_requires_forward_output'></a>this_layer_requires_forward_output</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#979000'>true</font>; <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='disable_output_and_gradient_getters'></a>disable_output_and_gradient_getters</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#009900'>// This should never happen because only inplace layers call
</font>            <font color='#009900'>// disable_output_and_gradient_getters(), however, putting a tag layer right
</font>            <font color='#009900'>// before an inplace layer basically means you don't want the following layer
</font>            <font color='#009900'>// to operate in place.  So the inplace layer should turn itself into an
</font>            <font color='#009900'>// out-of-place layer and not call disable_output_and_gradient_getters(). 
</font>            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>,"<font color='#CC0000'>This should never happen</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b> <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        subnet_type subnetwork;

        <font color='#009900'>// This member doesn't logically contribute to the state of the object since it is
</font>        <font color='#009900'>// always empty. It's just here so we can have the get_parameter_gradient() methods
</font>        <font color='#009900'>// which have to return something.  So they return this empty tensor.
</font>        resizable_tensor params_grad;
    <b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='decorator_repeat_group'></a>decorator_repeat_group</b>
    <b>{</b>
        <b><a name='decorator_repeat_group'></a>decorator_repeat_group</b><font face='Lucida Console'>(</font>
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : data<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        std::tuple<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font> data;
    <b>}</b>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
    decorator_repeat_group<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font> <b><a name='repeat_group'></a>repeat_group</b> <font face='Lucida Console'>(</font>
        T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>return</font> decorator_repeat_group<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>size_t</u></font> num,
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='REPEATED_LAYER'></a>REPEATED_LAYER</b>, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='repeat'></a>repeat</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>num <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't have a layer repeated 0 times.</font>"<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> SUBNET subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> SUBNET::input_type input_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>int</u></font> layer_details_type; <font color='#009900'>// not really used anywhere, but required by subnet_wrapper.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> comp_layers_in_each_group <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>REPEATED_LAYER<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font>::num_computational_layers<font color='#5555FF'>-</font>SUBNET::num_computational_layers<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> comp_layers_in_repeated_group <font color='#5555FF'>=</font> comp_layers_in_each_group<font color='#5555FF'>*</font>num;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> comp_layers_in_repeated_group <font color='#5555FF'>+</font> SUBNET::num_computational_layers;

        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> layers_in_each_group <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>REPEATED_LAYER<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font>::num_layers<font color='#5555FF'>-</font>SUBNET::num_layers<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> layers_in_repeated_group <font color='#5555FF'>=</font> layers_in_each_group<font color='#5555FF'>*</font>num;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> subnet_type::num_layers <font color='#5555FF'>+</font> layers_in_repeated_group;


        <font color='#0000FF'>typedef</font> REPEATED_LAYER<font color='#5555FF'>&lt;</font>impl::repeat_input_layer<font color='#5555FF'>&gt;</font> repeated_layer_type;

        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> : 
            details<font face='Lucida Console'>(</font>num<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>size_t</u></font> <b><a name='num_repetitions'></a>num_repetitions</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> num; <b>}</b>

        <font color='#0000FF'>const</font> repeated_layer_type<font color='#5555FF'>&amp;</font> <b><a name='get_repeated_layer'></a>get_repeated_layer</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>size_t</u></font> i 
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>num_repetitions</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> details[i]; 
        <b>}</b>

        repeated_layer_type<font color='#5555FF'>&amp;</font> <b><a name='get_repeated_layer'></a>get_repeated_layer</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>size_t</u></font> i 
        <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>num_repetitions</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> details[i]; 
        <b>}</b>

        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font>repeat<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        repeat<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>repeat<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        repeat<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>num,T,U<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : 
            subnetwork<font face='Lucida Console'>(</font>item.subnetwork<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> d : item.details<font face='Lucida Console'>)</font>
                details.<font color='#BB00BB'>emplace_back</font><font face='Lucida Console'>(</font>d<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font>
            T arg1,
            U ...args2
        <font face='Lucida Console'>)</font>: 
            details<font face='Lucida Console'>(</font>num, std::move<font face='Lucida Console'>(</font>arg1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            subnetwork<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>args2<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font>
            decorator_repeat_group<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> arg1,
            U ...args2
        <font face='Lucida Console'>)</font>: 
            details<font face='Lucida Console'>(</font>num, arg1.data<font face='Lucida Console'>)</font>,
            subnetwork<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>args2<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <b><a name='repeat'></a>repeat</b><font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>,
            T arg1,
            U ...args2
        <font face='Lucida Console'>)</font>: 
            details<font face='Lucida Console'>(</font>num, std::move<font face='Lucida Console'>(</font>arg1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            subnetwork<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>args2<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,data<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>temp_tensor<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x, <font color='#5555FF'>&amp;</font>x<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            details[details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>].<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>subnetwork.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>2</font>; i <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>; <font color='#5555FF'>-</font><font color='#5555FF'>-</font>i<font face='Lucida Console'>)</font>
                details[i].<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>details[i<font color='#5555FF'>+</font><font color='#979000'>1</font>].<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b> 
            <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; 
        <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>get_parameter_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>get_parameter_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x, <font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <b>{</b>
                details[<font color='#979000'>0</font>].<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>details[<font color='#979000'>1</font>].<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>1</font>; i <font color='#5555FF'>&lt;</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i<font color='#5555FF'>+</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        details[i].<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>details[i<font color='#5555FF'>+</font><font color='#979000'>1</font>].<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, details[i<font color='#5555FF'>-</font><font color='#979000'>1</font>].<font color='#BB00BB'>get_final_data_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>else</font>
                        details[i].<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>subnetwork.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, details[i<font color='#5555FF'>-</font><font color='#979000'>1</font>].<font color='#BB00BB'>get_final_data_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                details[<font color='#979000'>0</font>].<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>subnetwork.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
            <b>}</b>
            subnetwork.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x, details.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_final_data_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b><font face='Lucida Console'>(</font>sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> solvers, <font color='#0000FF'><u>double</u></font> learning_rate<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                details[i].<font color='#BB00BB'>update_parameters</font><font face='Lucida Console'>(</font>solvers.<font color='#BB00BB'>pop</font><font face='Lucida Console'>(</font>comp_layers_in_each_group<font color='#5555FF'>*</font>i<font face='Lucida Console'>)</font>,learning_rate<font face='Lucida Console'>)</font>;
            subnetwork.<font color='#BB00BB'>update_parameters</font><font face='Lucida Console'>(</font>solvers.<font color='#BB00BB'>pop</font><font face='Lucida Console'>(</font>comp_layers_in_each_group<font color='#5555FF'>*</font>details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,learning_rate<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            temp_tensor.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            subnetwork.<font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> d : details<font face='Lucida Console'>)</font>
                d.<font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.details, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.subnetwork, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>repeat<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::repeat.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.details, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.subnetwork, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> repeat<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>num_repetitions</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>get_repeated_layer</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, idx, min_length<font face='Lucida Console'>)</font>;
                idx <font color='#5555FF'>+</font><font color='#5555FF'>=</font> layers_in_each_group;
            <b>}</b>
            <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, idx, min_length<font face='Lucida Console'>)</font>;
        <b>}</b>
    <font color='#0000FF'>private</font>:


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='dimpl'></a>dimpl</b>::subnet_wrapper;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_tag_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_skip_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> repeat;

        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_requires_forward_output'></a>this_layer_requires_forward_output</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> details[<font color='#979000'>0</font>].<font color='#BB00BB'>this_layer_requires_forward_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; 
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='disable_output_and_gradient_getters'></a>disable_output_and_gradient_getters</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b> 
            details[<font color='#979000'>0</font>].<font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>


        std::vector<font color='#5555FF'>&lt;</font>repeated_layer_type<font color='#5555FF'>&gt;</font> details; 
        subnet_type subnetwork;

        <font color='#009900'>// temp_tensor doesn't logically contribute to the state of this class.
</font>        <font color='#009900'>// It is here only to void needing to reallocate it over and over.
</font>        resizable_tensor temp_tensor;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>size_t</u></font> num,
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='REPEATED_LAYER'></a>REPEATED_LAYER</b>, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_nonloss_layer_type'></a>is_nonloss_layer_type</b><font color='#5555FF'>&lt;</font>repeat<font color='#5555FF'>&lt;</font>num,REPEATED_LAYER,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#009900'>// This version of add_tag_layer handles the special case where the subnetwork being given
</font><font color='#009900'>// is just an input layer object.
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> ID, <font color='#0000FF'>typename</font> INPUT_LAYER, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_tag_layer'></a>add_tag_layer</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> INPUT_LAYER subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> subnet_type::input_type input_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>int</u></font> layer_details_type; <font color='#009900'>// not really used anywhere, but required by subnet_wrapper.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> ID;

        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>:cached_output_ptr<font face='Lucida Console'>(</font>nullptr<font face='Lucida Console'>)</font>,gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,_sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_tag_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> : add_tag_layer<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <b>}</b>
        add_tag_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&lt;</font>ID,T,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : input_layer<font face='Lucida Console'>(</font>item.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            cached_output<font face='Lucida Console'>(</font>item.cached_output<font face='Lucida Console'>)</font>,
            cached_output_ptr<font face='Lucida Console'>(</font>nullptr<font face='Lucida Console'>)</font>,
            grad_final<font face='Lucida Console'>(</font>item.grad_final<font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font>item.gradient_input_is_stale<font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_tag_layer'></a>add_tag_layer</b><font face='Lucida Console'>(</font>
            T ...args
        <font face='Lucida Console'>)</font> : 
            input_layer<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>,
            cached_output_ptr<font face='Lucida Console'>(</font>nullptr<font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <b><a name='add_tag_layer'></a>add_tag_layer</b> <font face='Lucida Console'>(</font>
            std::tuple<font color='#5555FF'>&lt;</font><font color='#5555FF'>&gt;</font>
        <font face='Lucida Console'>)</font> : 
            cached_output_ptr<font face='Lucida Console'>(</font>nullptr<font face='Lucida Console'>)</font>,
            gradient_input_is_stale<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>,
            _sample_expansion_factor<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            input_layer.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,data<font face='Lucida Console'>)</font>;

            <font color='#009900'>// make sure the input layer's to_tensor() function is implemented properly.
</font>            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The input layer can't produce fewer output tensors than there are inputs.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>,
            "<font color='#CC0000'>The number of tensors produced by the input layer must be an integer multiple of the number of input objects.</font>"<font face='Lucida Console'>)</font>;

            _sample_expansion_factor <font color='#5555FF'>=</font> data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font>;
            data.<font color='#BB00BB'>async_copy_to_device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _sample_expansion_factor; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin, 
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            input_layer.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,cached_output<font face='Lucida Console'>)</font>;
            cached_output_ptr <font color='#5555FF'>=</font> nullptr;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x, <font color='#5555FF'>&amp;</font>x<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// If this tag is the first layer in one of the sub networks inside a repeat
</font>            <font color='#009900'>// layer then we don't want it to be creating copies of x.  This is because, we
</font>            <font color='#009900'>// can just hold a pointer to x since the way repeat is constructed guarantees
</font>            <font color='#009900'>// that x will have a lifetime larger than this pointer. 
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>is_same_type<font color='#5555FF'>&lt;</font>INPUT_LAYER, impl::repeat_input_layer<font color='#5555FF'>&gt;</font>::value<font face='Lucida Console'>)</font>
                cached_output_ptr <font color='#5555FF'>=</font> <font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font>tensor<font color='#5555FF'>*</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                cached_output <font color='#5555FF'>=</font> x;
            gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>cached_output_ptr<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>cached_output_ptr;
            <font color='#0000FF'>else</font>
                <font color='#0000FF'>return</font> cached_output; 
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_final_data_gradient'></a>get_final_data_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> grad_final; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font><font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, grad_final<font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
                gradient_input_is_stale<font face='Lucida Console'>)</font>
            <b>{</b>
                grad_final.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                grad_final <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                gradient_input_is_stale <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> grad_final; 
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*x*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// nothing to do
</font>        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*x*/</font>, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*gradient_input*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// nothing to do
</font>        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b><font face='Lucida Console'>(</font>sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> <font color='#009900'>/*solvers*/</font>, <font color='#0000FF'><u>double</u></font> <font color='#009900'>/*learning_rate*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// nothing to do
</font>        <b>}</b>

        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> input_layer; <b>}</b>
        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> input_layer; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            grad_final.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cached_output.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cached_output_ptr <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.input_layer, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.cached_output, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.grad_final, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gradient_input_is_stale, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item._sample_expansion_factor, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> version <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> version <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::add_tag_layer.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.input_layer, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.cached_output, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.grad_final, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gradient_input_is_stale, in<font face='Lucida Console'>)</font>;
            item.cached_output_ptr <font color='#5555FF'>=</font> nullptr;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item._sample_expansion_factor, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                item._sample_expansion_factor <font color='#5555FF'>=</font> <font color='#979000'>1</font>; <font color='#009900'>// all layer types set this to 1 in older dlib versions, so that's what we put here.
</font>                
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_tag_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>idx <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>impl::<font color='#BB00BB'>tensor_to_str</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, min_length<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>tag</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> ID <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";
            <font color='#009900'>// Don't print the repeat_input_layer since it doesn't exist from the user's
</font>            <font color='#009900'>// point of view.  It's just an artifact of how repeat&lt;&gt; works.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>std::is_same<font color='#5555FF'>&lt;</font>subnet_type, impl::repeat_input_layer<font color='#5555FF'>&gt;</font>::value<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx<font color='#5555FF'>+</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='dimpl'></a>dimpl</b>::subnet_wrapper;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_tag_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_skip_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> repeat;

        <font color='#009900'>// You woudln't put a tag on a layer if you didn't want to access its forward
</font>        <font color='#009900'>// outputs.  So this is always true.
</font>        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_requires_forward_output'></a>this_layer_requires_forward_output</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#979000'>true</font>; <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='disable_output_and_gradient_getters'></a>disable_output_and_gradient_getters</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#009900'>// This should never happen because only inplace layers call
</font>            <font color='#009900'>// disable_output_and_gradient_getters(), however, putting a tag layer right
</font>            <font color='#009900'>// before an inplace layer basically means you don't want the following layer
</font>            <font color='#009900'>// to operate in place.  So the inplace layer should turn itself into an
</font>            <font color='#009900'>// out-of-place layer and not call disable_output_and_gradient_getters(). 
</font>            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>,"<font color='#CC0000'>This should never happen</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b> <font color='#0000FF'>return</font> <font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font>tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='swap'></a>swap</b><font face='Lucida Console'>(</font>add_tag_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>input_layer, item.input_layer<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>cached_output, item.cached_output<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>cached_output_ptr, item.cached_output_ptr<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>grad_final, item.grad_final<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>gradient_input_is_stale, item.gradient_input_is_stale<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>_sample_expansion_factor, item._sample_expansion_factor<font face='Lucida Console'>)</font>;
        <b>}</b>

        subnet_type input_layer;
        resizable_tensor cached_output;
        tensor<font color='#5555FF'>*</font> cached_output_ptr;
        resizable_tensor grad_final;
        <font color='#0000FF'><u>bool</u></font> gradient_input_is_stale;
        <font color='#0000FF'>mutable</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> _sample_expansion_factor;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> ID, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_nonloss_layer_type'></a>is_nonloss_layer_type</b><font color='#5555FF'>&lt;</font>add_tag_layer<font color='#5555FF'>&lt;</font>ID,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;


<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> LOSS_DETAILS, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> add_loss_layer;

    <font color='#0000FF'>class</font> <b><a name='no_label_type'></a>no_label_type</b>
    <b>{</b>
    <font color='#0000FF'>private</font>:
        <font color='#009900'>// We don't want anyone making these no_label_type objects.  They are here only to
</font>        <font color='#009900'>// allow add_loss_layer::label_type and dnn_trainer::label_type to exist which avoids
</font>        <font color='#009900'>// needing to overload add_loss_layer and dnn_trainer for supervised an unsupervised
</font>        <font color='#009900'>// losses.  It also can be a type to use in template metaprogramming to indicate
</font>        <font color='#009900'>// "no label".  So here we make the constructor private with the exception that
</font>        <font color='#009900'>// add_loss_layer objects can make it (again, just to simplify add_loss_layer's
</font>        <font color='#009900'>// implementation).
</font>        <b><a name='no_label_type'></a>no_label_type</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><b>{</b><b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> LOSS_DETAILS, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_loss_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font> <font color='#0000FF'>typename</font> net_type, <font color='#0000FF'>typename</font> solver_type <font color='#5555FF'>&gt;</font> <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> dnn_trainer; 
    <b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> LOSS_DETAILS, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_loss_layer'></a>add_loss_layer</b>
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>=</font><font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='get_loss_layer_label_type'></a>get_loss_layer_label_type</b>
        <b>{</b>
            <font color='#0000FF'>typedef</font> no_label_type type;
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='get_loss_layer_label_type'></a>get_loss_layer_label_type</b><font color='#5555FF'>&lt;</font>T,<font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font color='#0000FF'>sizeof</font><font face='Lucida Console'>(</font><font color='#0000FF'>typename</font> T::label_type<font face='Lucida Console'>)</font><font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> T::label_type type;
        <b>}</b>;

    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> LOSS_DETAILS loss_details_type;
        <font color='#0000FF'>typedef</font> SUBNET subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> subnet_type::input_type input_type;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> subnet_type::num_layers <font color='#5555FF'>+</font> <font color='#979000'>1</font>;
        <font color='#009900'>// Note that the loss layer doesn't count as an additional computational layer.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> subnet_type::num_computational_layers;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> get_loss_layer_label_type<font color='#5555FF'>&lt;</font>LOSS_DETAILS<font color='#5555FF'>&gt;</font>::type label_type;

        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>is_nonloss_layer_type<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font>::value, 
            "<font color='#CC0000'>SUBNET must be of type add_layer, add_skip_layer, or add_tag_layer.</font>"<font face='Lucida Console'>)</font>; 


        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>;
        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_loss_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_loss_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_loss_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font>add_loss_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> : add_loss_layer<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <b>}</b>
        add_loss_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>add_loss_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> add_loss_layer<font color='#5555FF'>&lt;</font>T,U<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : 
            loss<font face='Lucida Console'>(</font>item.loss_details<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
            subnetwork<font face='Lucida Console'>(</font>item.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> LOSS_DETAILS<font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            loss<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font>, 
            subnetwork<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font>
            LOSS_DETAILS<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> layer_det, 
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            loss<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>layer_det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            subnetwork<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> ...U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b> 
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> std::is_constructible<font color='#5555FF'>&lt;</font>LOSS_DETAILS,T<font color='#5555FF'>&gt;</font>::value;
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='disable_forwarding_constr'></a>disable_forwarding_constr</b><font color='#5555FF'>&lt;</font>add_loss_layer<font color='#5555FF'>&lt;</font>T...<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> value <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> ...T, 
            <font color='#0000FF'>typename</font> <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font color='#5555FF'>!</font>disable_forwarding_constr<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font>::type...<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type
            <font color='#5555FF'>&gt;</font>
        <b><a name='add_loss_layer'></a>add_loss_layer</b><font face='Lucida Console'>(</font>
            T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> ...args
        <font face='Lucida Console'>)</font> : 
            subnetwork<font face='Lucida Console'>(</font>std::forward<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,data<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> output_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x, 
            output_iterator obegin
        <font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>;
            loss.<font color='#BB00BB'>to_label</font><font face='Lucida Console'>(</font>x, wsub, obegin<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator, <font color='#0000FF'>typename</font> output_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            output_iterator obegin
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>temp_tensor, obegin<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> label_type<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x, <font color='#5555FF'>&amp;</font>x<font color='#5555FF'>+</font><font color='#979000'>1</font>, <font color='#5555FF'>&amp;</font>temp_label<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> temp_label;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> iterable_type<font color='#5555FF'>&gt;</font>
        std::vector<font color='#5555FF'>&lt;</font>label_type<font color='#5555FF'>&gt;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> iterable_type<font color='#5555FF'>&amp;</font> data,
            <font color='#0000FF'><u>size_t</u></font> batch_size <font color='#5555FF'>=</font> <font color='#979000'>128</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            std::vector<font color='#5555FF'>&lt;</font>label_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>results</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>distance</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, data.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> o <font color='#5555FF'>=</font> results.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> i <font color='#5555FF'>=</font> data.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> num_remaining <font color='#5555FF'>=</font> results.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>while</font><font face='Lucida Console'>(</font>num_remaining <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> inc <font color='#5555FF'>=</font> std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font>batch_size, num_remaining<font face='Lucida Console'>)</font>;
                <font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>i, i<font color='#5555FF'>+</font>inc, o<font face='Lucida Console'>)</font>;
                i <font color='#5555FF'>+</font><font color='#5555FF'>=</font> inc;
                o <font color='#5555FF'>+</font><font color='#5555FF'>=</font> inc;
                num_remaining <font color='#5555FF'>-</font><font color='#5555FF'>=</font> inc;
            <b>}</b>
            <font color='#0000FF'>return</font> results;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> label_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss'></a>compute_loss</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x,
            label_iterator lbegin 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> loss.<font color='#BB00BB'>compute_loss_value_and_gradient</font><font face='Lucida Console'>(</font>x, lbegin, wsub<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator, <font color='#0000FF'>typename</font> label_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss'></a>compute_loss</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            label_iterator lbegin 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>compute_loss</font><font face='Lucida Console'>(</font>temp_tensor, lbegin<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss'></a>compute_loss</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x
        <font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> loss.<font color='#BB00BB'>compute_loss_value_and_gradient</font><font face='Lucida Console'>(</font>x, wsub<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss'></a>compute_loss</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>compute_loss</font><font face='Lucida Console'>(</font>temp_tensor<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> label_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_parameter_gradients'></a>compute_parameter_gradients</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x,
            label_iterator lbegin
        <font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> l <font color='#5555FF'>=</font> loss.<font color='#BB00BB'>compute_loss_value_and_gradient</font><font face='Lucida Console'>(</font>x, lbegin, wsub<font face='Lucida Console'>)</font>;
            subnetwork.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> l;
        <b>}</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator, <font color='#0000FF'>typename</font> label_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_parameter_gradients'></a>compute_parameter_gradients</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            label_iterator lbegin
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>compute_parameter_gradients</font><font face='Lucida Console'>(</font>temp_tensor, lbegin<font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_parameter_gradients'></a>compute_parameter_gradients</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x
        <font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>subnet_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> l <font color='#5555FF'>=</font> loss.<font color='#BB00BB'>compute_loss_value_and_gradient</font><font face='Lucida Console'>(</font>x, wsub<font face='Lucida Console'>)</font>;
            subnetwork.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> l;
        <b>}</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_parameter_gradients'></a>compute_parameter_gradients</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,temp_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>compute_parameter_gradients</font><font face='Lucida Console'>(</font>temp_tensor<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b> <font face='Lucida Console'>(</font>
            sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> solvers,
            <font color='#0000FF'><u>double</u></font> learning_rate
        <font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>update_parameters</font><font face='Lucida Console'>(</font>solvers, learning_rate<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> subnetwork; <b>}</b>
        <font color='#0000FF'>const</font> loss_details_type<font color='#5555FF'>&amp;</font> <b><a name='loss_details'></a>loss_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> loss; <b>}</b>
        loss_details_type<font color='#5555FF'>&amp;</font> <b><a name='loss_details'></a>loss_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> loss; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            temp_tensor.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            subnetwork.<font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_loss_layer<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.loss, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.subnetwork, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_loss_layer<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::add_loss_layer.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.loss, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.subnetwork, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_loss_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>loss_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>";
            <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, idx<font color='#5555FF'>+</font><font color='#979000'>1</font>, min_length<font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:


        <font color='#0000FF'><u>void</u></font> <b><a name='swap'></a>swap</b><font face='Lucida Console'>(</font>add_loss_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>loss, item.loss<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>subnetwork, item.subnetwork<font face='Lucida Console'>)</font>;
        <b>}</b>

        loss_details_type loss;
        subnet_type subnetwork;

        <font color='#009900'>// These two objects don't logically contribute to the state of this object.  They
</font>        <font color='#009900'>// are here to prevent them from being reallocated over and over.
</font>        label_type temp_label;
        resizable_tensor temp_tensor;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_loss_layer_type'></a>is_loss_layer_type</b><font color='#5555FF'>&lt;</font>add_loss_layer<font color='#5555FF'>&lt;</font>T,U<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i, <font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> enabled <font color='#5555FF'>=</font> <font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b>
        <b>{</b>
            <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> T::num_layers, "<font color='#CC0000'>Call to layer() attempted to access non-existing layer in neural network.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>static</font> T<font color='#5555FF'>&amp;</font> <b><a name='makeT'></a>makeT</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>using</font> next_type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font><font color='#BB00BB'>makeT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>-</font><font color='#979000'>1</font>,next_type<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>-</font><font color='#979000'>1</font>,next_type<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i,
            <font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font>i,repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>, <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>i<font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_repeated_group<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> layers_in_repeated_group <font color='#5555FF'>=</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_repeated_group;

            <font color='#0000FF'>static</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>makeT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>using</font> next_type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font><font color='#BB00BB'>decltype</font><font face='Lucida Console'>(</font><font color='#BB00BB'>makeT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>-</font>layers_in_repeated_group,next_type<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>-</font>layers_in_repeated_group,next_type<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i,
            <font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font>i,repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>, <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>i<font color='#5555FF'>&lt;</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_repeated_group<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> layers_in_each_group <font color='#5555FF'>=</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_each_group;
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::repeated_layer_type repeated_layer_type;
            <font color='#0000FF'>using</font> next_type <font color='#5555FF'>=</font> repeated_layer_type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>%</font>layers_in_each_group,next_type<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>%</font>layers_in_each_group,next_type<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n.<font color='#BB00BB'>get_repeated_layer</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>/</font>layers_in_each_group<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>, <font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::repeated_layer_type repeated_layer_type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> repeated_layer_type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> n.<font color='#BB00BB'>get_repeated_layer</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;



        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i,
            <font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font>i,<font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>, <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>i<font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_repeated_group<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> layers_in_repeated_group <font color='#5555FF'>=</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_repeated_group;

            <font color='#0000FF'>static</font> <font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> <font color='#BB00BB'>makeT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>using</font> next_type <font color='#5555FF'>=</font> <font color='#0000FF'>const</font> <font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font><font color='#BB00BB'>decltype</font><font face='Lucida Console'>(</font><font color='#BB00BB'>makeT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>const</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>-</font>layers_in_repeated_group,next_type<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>-</font>layers_in_repeated_group,next_type<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i,
            <font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font>i,<font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>, <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>i<font color='#5555FF'>&lt;</font>repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_repeated_group<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> layers_in_each_group <font color='#5555FF'>=</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::layers_in_each_group;
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::repeated_layer_type repeated_layer_type;
            <font color='#0000FF'>using</font> next_type <font color='#5555FF'>=</font> <font color='#0000FF'>const</font> repeated_layer_type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>const</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>%</font>layers_in_each_group,next_type<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>%</font>layers_in_each_group,next_type<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n.<font color='#BB00BB'>get_repeated_layer</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>/</font>layers_in_each_group<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>, <font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font>::repeated_layer_type repeated_layer_type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>const</font> repeated_layer_type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> repeat<font color='#5555FF'>&lt;</font>N,L,S<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> n.<font color='#BB00BB'>get_repeated_layer</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;



        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper'></a>layer_helper</b><font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,T,<font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> T;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> n;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='Match'></a>Match</b>, <font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i, <font color='#0000FF'>typename</font> enabled <font color='#5555FF'>=</font> <font color='#0000FF'><u>void</u></font><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper_match'></a>layer_helper_match</b>
        <b>{</b>
            <font color='#0000FF'>static</font> T<font color='#5555FF'>&amp;</font> <b><a name='makeT'></a>makeT</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>using</font> next_type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> std::remove_reference<font color='#5555FF'>&lt;</font><b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font><font color='#BB00BB'>makeT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper_match<font color='#5555FF'>&lt;</font>Match,next_type,i<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper_match<font color='#5555FF'>&lt;</font>Match,next_type,i<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#009900'>// This overload catches add_layer and add_loss_layer templates.
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='Match'></a>Match</b>, <font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper_match'></a>layer_helper_match</b><font color='#5555FF'>&lt;</font>Match,T,i,
            <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>std::is_same<font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> T,<font color='#0000FF'>const</font>  Match<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::subnet_type<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#009900'>// This overload catches input templates.
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='Match'></a>Match</b>, <font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper_match'></a>layer_helper_match</b><font color='#5555FF'>&lt;</font>Match,T,i,
            <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>std::is_same<font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> T,<font color='#0000FF'>const</font>  Match<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::input_type<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#009900'>// This overload catches subnet_wrapper templates.
</font>        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='Match'></a>Match</b>, <font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='layer_helper_match'></a>layer_helper_match</b><font color='#5555FF'>&lt;</font>Match,T,i,
            <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>std::is_same<font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> <font color='#0000FF'>typename</font> T::wrapped_type, 
                                                 <font color='#0000FF'>const</font> Match<font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T::wrapped_type::subnet_type<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>using</font> type <font color='#5555FF'>=</font> <font color='#0000FF'>typename</font> layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::type;
            <font color='#0000FF'>static</font> type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i, <font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>typename</font> impl::layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b> <font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font> 
    <b>{</b>
        <font color='#0000FF'>return</font> impl::layer_helper<font color='#5555FF'>&lt;</font>i,T<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='Match'></a>Match</b>, <font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>typename</font> impl::layer_helper_match<font color='#5555FF'>&lt;</font>Match,T,<font color='#979000'>0</font><font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b> <font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font> 
    <b>{</b>
        <font color='#0000FF'>return</font> impl::layer_helper_match<font color='#5555FF'>&lt;</font>Match,T,<font color='#979000'>0</font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='Match'></a>Match</b>, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> i, <font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>typename</font> impl::layer_helper_match<font color='#5555FF'>&lt;</font>Match,T,i<font color='#5555FF'>&gt;</font>::type<font color='#5555FF'>&amp;</font> <b><a name='layer'></a>layer</b> <font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font> n<font face='Lucida Console'>)</font> 
    <b>{</b>
        <font color='#0000FF'>return</font> impl::layer_helper_match<font color='#5555FF'>&lt;</font>Match,T,i<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>layer</font><font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>

    <font color='#0000FF'>namespace</font> dimpl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        T<font color='#5555FF'>&amp;</font> <b><a name='get_input_details'></a>get_input_details</b> <font face='Lucida Console'>(</font>
            T<font color='#5555FF'>&amp;</font> net
        <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> net; 
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='get_input_details'></a>get_input_details</b> <font face='Lucida Console'>(</font>
            dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>T,is_first,enabled<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> net
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&amp;</font>
        <b>{</b>
            <font color='#0000FF'>return</font> net.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> enabled<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>auto</font> <b><a name='get_input_details'></a>get_input_details</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>T,is_first,enabled<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> net
        <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&amp;</font>
        <b>{</b>
            <font color='#0000FF'>return</font> net.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> net_type<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>auto</font> <b><a name='input_layer'></a>input_layer</b> <font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net
    <font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font> <b><a name='decltype'></a>decltype</b><font face='Lucida Console'>(</font>dimpl::<font color='#BB00BB'>get_input_details</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>net_type::num_layers<font color='#5555FF'>-</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>&amp;</font>
    <b>{</b>
        <font color='#009900'>// Calling input_layer() on a subnet_wrapper is a little funny since the behavior of
</font>        <font color='#009900'>// .subnet() returns another subnet_wrapper rather than an input details object as it
</font>        <font color='#009900'>// does in add_layer.
</font>        <font color='#0000FF'>return</font> dimpl::<font color='#BB00BB'>get_input_details</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>net_type::num_layers<font color='#5555FF'>-</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b>, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_skip_layer'></a>add_skip_layer</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> SUBNET subnet_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> subnet_type::input_type input_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>int</u></font> layer_details_type; <font color='#009900'>// not really used anywhere, but required by subnet_wrapper.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_layers <font color='#5555FF'>=</font> subnet_type::num_layers <font color='#5555FF'>+</font> <font color='#979000'>1</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> num_computational_layers <font color='#5555FF'>=</font> subnet_type::num_computational_layers;
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id;

        <b><a name='add_skip_layer'></a>add_skip_layer</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>;
        <b><a name='add_skip_layer'></a>add_skip_layer</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_skip_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='add_skip_layer'></a>add_skip_layer</b><font face='Lucida Console'>(</font>add_skip_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_skip_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font>add_skip_layer<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        add_skip_layer<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_skip_layer<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <b><a name='add_skip_layer'></a>add_skip_layer</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> add_skip_layer<font color='#5555FF'>&lt;</font>TAG_TYPE,T<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : subnetwork<font face='Lucida Console'>(</font>item.subnet<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> ...T<font color='#5555FF'>&gt;</font>
        <b><a name='add_skip_layer'></a>add_skip_layer</b><font face='Lucida Console'>(</font>
            T ...args
        <font face='Lucida Console'>)</font> : 
            subnetwork<font face='Lucida Console'>(</font>std::move<font face='Lucida Console'>(</font>args<font face='Lucida Console'>)</font>...<font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_tensor'></a>to_tensor</b> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend,
            resizable_tensor<font color='#5555FF'>&amp;</font> data
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font>ibegin,iend,data<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> forward_iterator<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
            forward_iterator ibegin,
            forward_iterator iend
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>subnetwork</font><font face='Lucida Console'>(</font>ibegin,iend<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> input_type<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>subnetwork</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_final_data_gradient'></a>get_final_data_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> subnetwork.<font color='#BB00BB'>get_final_data_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; 
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='back_propagate_error'></a>back_propagate_error</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> solver_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='update_parameters'></a>update_parameters</b><font face='Lucida Console'>(</font>sstack<font color='#5555FF'>&lt;</font>solver_type<font color='#5555FF'>&gt;</font> solvers, <font color='#0000FF'><u>double</u></font> learning_rate<font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>update_parameters</font><font face='Lucida Console'>(</font>solvers, learning_rate<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_parameter_gradient'></a>get_parameter_gradient</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params_grad; <b>}</b>


        <font color='#0000FF'>const</font> subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> subnetwork; 
        <b>}</b>

        subnet_type<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#0000FF'>return</font> subnetwork; 
        <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> <b><a name='sample_expansion_factor'></a>sample_expansion_factor</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            subnetwork.<font color='#BB00BB'>clean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_skip_layer<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.subnetwork, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_skip_layer<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::add_skip_layer.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.subnetwork, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_skip_layer<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>int</u></font> min_length <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            item.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, <font color='#979000'>0</font>, min_length<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='print'></a>print</b> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx, <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&amp;</font> min_length<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer&lt;</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> idx <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\t</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>impl::<font color='#BB00BB'>tensor_to_str</font><font face='Lucida Console'>(</font><font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, min_length<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>skip</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n</font>";
            <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>print</font><font face='Lucida Console'>(</font>out, idx<font color='#5555FF'>+</font><font color='#979000'>1</font>, min_length<font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'><u>bool</u></font> is_first, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='dimpl'></a>dimpl</b>::subnet_wrapper;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_tag_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> add_skip_layer;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> N, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='L'></a>L</b>, <font color='#0000FF'>typename</font> S<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> repeat;

        <font color='#0000FF'><u>bool</u></font> <b><a name='this_layer_requires_forward_output'></a>this_layer_requires_forward_output</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>this_layer_requires_forward_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='disable_output_and_gradient_getters'></a>disable_output_and_gradient_getters</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <b>{</b> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>disable_output_and_gradient_getters</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b> <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>private_get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b> <font color='#0000FF'>return</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>.<font color='#BB00BB'>private_get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        subnet_type subnetwork;

        <font color='#009900'>// This member doesn't logically contribute to the state of the object since it is
</font>        <font color='#009900'>// always empty. It's just here so we can have the get_parameter_gradient() methods
</font>        <font color='#009900'>// which have to return something.  So they return this empty tensor.
</font>        resizable_tensor params_grad;
    <b>}</b>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='T'></a>T</b>, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>struct</font> <b><a name='is_nonloss_layer_type'></a>is_nonloss_layer_type</b><font color='#5555FF'>&lt;</font>add_skip_layer<font color='#5555FF'>&lt;</font>T,U<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> : std::true_type <b>{</b><b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag1  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag2  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>2</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag3  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>3</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag4  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>4</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag5  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>5</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag6  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>6</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag7  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>7</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag8  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>8</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag9  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>9</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> tag10 <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip1  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag1, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip2  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag2, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip3  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag3, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip4  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag4, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip5  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag5, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip6  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag6, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip7  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag7, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip8  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag8, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip9  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> tag9, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> skip10 <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font>tag10, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> timpl
    <b>{</b>
        <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='fill_with_gassuan_random_numbers'></a>fill_with_gassuan_random_numbers</b> <font face='Lucida Console'>(</font>
            tensor<font color='#5555FF'>&amp;</font> t,
            dlib::rand<font color='#5555FF'>&amp;</font> rnd,
            <font color='#0000FF'><u>double</u></font> sigma <font color='#5555FF'>=</font> <font color='#979000'>1</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> data <font color='#5555FF'>=</font> t.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> t.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                data[i] <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_gaussian</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sigma;
        <b>}</b>

        <font color='#0000FF'>class</font> <b><a name='test_layer_subnet'></a>test_layer_subnet</b> 
        <b>{</b>
        <font color='#0000FF'>public</font>:
            <b><a name='test_layer_subnet'></a>test_layer_subnet</b> <font face='Lucida Console'>(</font>
                dlib::rand<font color='#5555FF'>&amp;</font> rnd_
            <font face='Lucida Console'>)</font> : rnd<font face='Lucida Console'>(</font>rnd_<font face='Lucida Console'>)</font> 
            <b>{</b>
                <font color='#009900'>// Output and gradient_input have to have the same dimensions in each
</font>                <font color='#009900'>// layer.
</font>                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_samples <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>3</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k  <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>2</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>2</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>2</font>;

                output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_samples, k, nr, nc<font face='Lucida Console'>)</font>;
                gradient_input.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_samples, k, nr, nc<font face='Lucida Console'>)</font>;

                <font color='#009900'>// Use a non-zero initial gradient to make sure the layers add to it
</font>                <font color='#009900'>// rather than assign and blow away the initial value.
</font>                <font color='#BB00BB'>fill_with_gassuan_random_numbers</font><font face='Lucida Console'>(</font>gradient_input, rnd, <font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;

                <font color='#BB00BB'>fill_with_gassuan_random_numbers</font><font face='Lucida Console'>(</font>output, rnd<font face='Lucida Console'>)</font>;
            <b>}</b>


            tensor<font color='#5555FF'>&amp;</font> <b><a name='get_mutable_output'></a>get_mutable_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> output; <b>}</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_output'></a>get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> output; <b>}</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_output'></a>private_get_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>
            <font color='#0000FF'>const</font> test_layer_subnet<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#BB00BB'>init_sub</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>subnetwork; <b>}</b>

            tensor<font color='#5555FF'>&amp;</font> <b><a name='get_gradient_input'></a>get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> gradient_input; <b>}</b>
            tensor<font color='#5555FF'>&amp;</font> <b><a name='private_get_gradient_input'></a>private_get_gradient_input</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>
            test_layer_subnet<font color='#5555FF'>&amp;</font> <b><a name='subnet'></a>subnet</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#BB00BB'>init_sub</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>subnetwork; <b>}</b>



            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='count_outputs'></a>count_outputs</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>return</font> subnetwork<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>count_outputs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    <font color='#0000FF'>return</font> output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>&amp;</font> <b><a name='get_output_element'></a>get_output_element</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <font color='#0000FF'>return</font> output.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i];
                <font color='#0000FF'>else</font>
                    <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output_element</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>-</font>output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'><u>float</u></font> <b><a name='get_gradient_input_element'></a>get_gradient_input_element</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <font color='#0000FF'>return</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i];
                <font color='#0000FF'>else</font>
                    <font color='#0000FF'>return</font> <font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input_element</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>-</font>gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>


        <font color='#0000FF'>private</font>:
            <font color='#009900'>// We lazily initialize sub-layers as needed when someone tries to call
</font>            <font color='#009900'>// subnet()
</font>            <font color='#0000FF'><u>void</u></font> <b><a name='init_sub'></a>init_sub</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>subnetwork<font face='Lucida Console'>)</font>
                    subnetwork.<font color='#BB00BB'>reset</font><font face='Lucida Console'>(</font><font color='#0000FF'>new</font> <font color='#BB00BB'>test_layer_subnet</font><font face='Lucida Console'>(</font>rnd<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            dlib::rand<font color='#5555FF'>&amp;</font> rnd;
            <font color='#0000FF'>mutable</font> std::unique_ptr<font color='#5555FF'>&lt;</font>test_layer_subnet<font color='#5555FF'>&gt;</font> subnetwork;
            resizable_tensor output;
            resizable_tensor gradient_input;
        <b>}</b>;

    <b>}</b>

    <font color='#0000FF'>struct</font> <b><a name='layer_test_results'></a>layer_test_results</b>
    <b>{</b>
        <b><a name='layer_test_results'></a>layer_test_results</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : was_good<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>
        <font color='#0000FF'>explicit</font> <b><a name='layer_test_results'></a>layer_test_results</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> l<font face='Lucida Console'>)</font> : log<font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>,was_good<font face='Lucida Console'>(</font><font color='#979000'>false</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        std::string log;
        <font color='#0000FF'><u>bool</u></font> was_good;

        <b><a name='operator'></a>operator</b> <font color='#0000FF'><u>bool</u></font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> was_good; <b>}</b>
    <b>}</b>;

    <font color='#0000FF'>inline</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> layer_test_results<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
    <b>{</b>
        out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.log;
        <font color='#0000FF'>return</font> out;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> layer_details_type
        <font color='#5555FF'>&gt;</font>
    layer_test_results <b><a name='impl_test_layer'></a>impl_test_layer</b> <font face='Lucida Console'>(</font>
        layer_details_type l,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> base_eps 
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> timpl;
        <font color='#009900'>// Do some setup
</font>        running_stats<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> rs_data, rs_params;
        dlib::rand rnd;
        std::ostringstream sout;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>10</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            test_layer_subnet <font color='#BB00BB'>subnetwork</font><font face='Lucida Console'>(</font>rnd<font face='Lucida Console'>)</font>;
            resizable_tensor output, out2, out3;
            <font color='#009900'>// Run setup() and forward() as well to make sure any calls to subnet() have
</font>            <font color='#009900'>// happened before we start assuming we know how many data elements there are
</font>            <font color='#009900'>// (since we do a lazy layer creation thing based on calls to subnet() inside
</font>            <font color='#009900'>// test_layer_subnet).
</font>            l.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>subnetwork<font face='Lucida Console'>)</font>;
            impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>l, subnetwork, output<font face='Lucida Console'>)</font>;

            resizable_tensor input_grad;
            input_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>output<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>fill_with_gassuan_random_numbers</font><font face='Lucida Console'>(</font>input_grad, rnd<font face='Lucida Console'>)</font>;


            <font color='#009900'>// The f() we are computing gradients of is this thing.  It's value at the current
</font>            <font color='#009900'>// parameter and data values is:
</font>            <font color='#009900'>//sout &lt;&lt; "f(data,params): " &lt;&lt; dot(output, input_grad) &lt;&lt; std::endl;
</font>
            <font color='#009900'>// We are going to save a copy of the subnetwork.get_gradient_input() data before we do
</font>            <font color='#009900'>// backpropagation since the backward() function is supposed to *add* to the
</font>            <font color='#009900'>// gradients rather than overwrite them.  We will use this saved data to check if
</font>            <font color='#009900'>// that is the case.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_data_inputs <font color='#5555FF'>=</font> subnetwork.<font color='#BB00BB'>count_outputs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>initial_gradient_input</font><font face='Lucida Console'>(</font>num_data_inputs<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num_data_inputs; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                initial_gradient_input[i] <font color='#5555FF'>=</font> subnetwork.<font color='#BB00BB'>get_gradient_input_element</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;


            <font color='#009900'>// Now tell the layer to compute all the gradients.  In the rest of this function
</font>            <font color='#009900'>// we will just be checking that these gradients were computed correctly by
</font>            <font color='#009900'>// comparing them to a central differences approximation.
</font>            resizable_tensor params_grad;
            params_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>l.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// But first, set the params grad to something crazy so that it's very obvious if
</font>            <font color='#009900'>// it doesn't get fully assigned.
</font>            params_grad <font color='#5555FF'>=</font> std::numeric_limits<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>infinity</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            impl::<font color='#BB00BB'>call_layer_backward</font><font face='Lucida Console'>(</font>l, output, input_grad, subnetwork, params_grad<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>impl::<font color='#BB00BB'>is_inplace_layer</font><font face='Lucida Console'>(</font>l, subnetwork<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> impl::<font color='#BB00BB'>has_inplace_backward</font><font face='Lucida Console'>(</font>l, subnetwork<font face='Lucida Console'>)</font>,
                "<font color='#CC0000'>Layer not defined correctly.  forward and backward methods must either both be in-place or both out-of-place. </font>"<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Make sure the outputs of forward() and backward() are the same when they are run
</font>            <font color='#009900'>// in in-place mode.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>impl::<font color='#BB00BB'>is_inplace_layer</font><font face='Lucida Console'>(</font>l, subnetwork<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                test_layer_subnet <font color='#BB00BB'>subnetwork2</font><font face='Lucida Console'>(</font>rnd<font face='Lucida Console'>)</font>;
                layer_details_type <font color='#BB00BB'>ll</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
                ll.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>subnetwork2<font face='Lucida Console'>)</font>;
                resizable_tensor ip_out;
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>ll, subnetwork2, ip_out<font face='Lucida Console'>)</font>;
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>ll, subnetwork2, subnetwork2.<font color='#BB00BB'>get_mutable_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> forward_error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>ip_out<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>subnetwork2.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>forward_error <font color='#5555FF'>&gt;</font> <font color='#979000'>0.00001</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>This layer is supposed to support in-place computations but the output of forward_inplace()\n</font>";
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>changes when invoked in-place vs. out-of-place. The error was: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> forward_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
                <b>}</b>

                resizable_tensor params_grad;
                params_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>ll.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                params_grad <font color='#5555FF'>=</font> std::numeric_limits<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>infinity</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                resizable_tensor input_grad;
                input_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>ip_out<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>fill_with_gassuan_random_numbers</font><font face='Lucida Console'>(</font>input_grad, rnd<font face='Lucida Console'>)</font>;
                resizable_tensor params_grad1, params_grad2, data_grad1, data_grad2;
                params_grad1 <font color='#5555FF'>=</font> params_grad;
                params_grad2 <font color='#5555FF'>=</font> params_grad;
                <font color='#009900'>// Now call backward() and make sure it works as well.  Recall that when an
</font>                <font color='#009900'>// in-place layer works in-place it assigns to it's outputs but when it's
</font>                <font color='#009900'>// not running in-place it adds.  So we initialize to a non-zero value to
</font>                <font color='#009900'>// check that this is the behavior that really executes.
</font>                subnetwork2.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>9</font>;
                impl::<font color='#BB00BB'>call_layer_backward</font><font face='Lucida Console'>(</font>ll, ip_out, input_grad, subnetwork2, params_grad1<font face='Lucida Console'>)</font>;
                data_grad1 <font color='#5555FF'>=</font> subnetwork2.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                subnetwork2.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>input_grad<font face='Lucida Console'>)</font>;
                impl::<font color='#BB00BB'>call_layer_backward</font><font face='Lucida Console'>(</font>ll, ip_out, subnetwork2.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, subnetwork2, params_grad2<font face='Lucida Console'>)</font>;
                data_grad2 <font color='#5555FF'>=</font> subnetwork2.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>params_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> backward_param_error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>params_grad1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>params_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>backward_param_error <font color='#5555FF'>&gt;</font> <font color='#979000'>0.00001</font><font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
                        sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>This layer is supposed to support in-place computations but the output of backward_inplace()\n</font>";
                        sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>changes when invoked in-place vs. out-of-place. The error was: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> backward_param_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                        <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
                    <b>}</b>
                <b>}</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> backward_data_error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_grad1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>9</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>backward_data_error <font color='#5555FF'>&gt;</font> <font color='#979000'>0.00001</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>This layer is supposed to support in-place computations but the output of backward_inplace()\n</font>";
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>changes when invoked in-place vs. out-of-place. The error was: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> backward_data_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// ==================================================================
</font>            <font color='#009900'>// first validate the way the parameter gradients are computed
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> params_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                layer_details_type <font color='#BB00BB'>l1</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;

                <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> l1.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i]<font color='#5555FF'>*</font>base_eps;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>eps <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    eps <font color='#5555FF'>=</font> base_eps;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> oldval <font color='#5555FF'>=</font> l1.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i];
                l1.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i] <font color='#5555FF'>=</font> oldval<font color='#5555FF'>+</font>eps;
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>l1, subnetwork, out2<font face='Lucida Console'>)</font>;
                l1.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i] <font color='#5555FF'>=</font> oldval<font color='#5555FF'>-</font>eps;
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>l1, subnetwork, out3<font face='Lucida Console'>)</font>;
                l1.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i] <font color='#5555FF'>=</font> oldval;

                <font color='#009900'>// Compute a reference derivative via a central differences approximation and
</font>                <font color='#009900'>// compare it to the one output by the layer and make sure they match.
</font>                <font color='#0000FF'><u>double</u></font> reference_derivative <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out2,input_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out3, input_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>double</u></font> output_derivative <font color='#5555FF'>=</font> params_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[i];
                <font color='#0000FF'><u>double</u></font> relative_error;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>reference_derivative<font color='#5555FF'>*</font>output_derivative <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    relative_error <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>reference_derivative <font color='#5555FF'>-</font> output_derivative<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>reference_derivative<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    relative_error <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>reference_derivative <font color='#5555FF'>-</font> output_derivative<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>double</u></font> absolute_error <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>reference_derivative <font color='#5555FF'>-</font> output_derivative<font face='Lucida Console'>)</font>;
                rs_params.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>relative_error<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>relative_error<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.05</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>absolute_error<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.006</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Gradient error in parameter #</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> i <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>.  Relative error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> relative_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>expected derivative: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> reference_derivative <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>output derivative:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_derivative <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>iteration:           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> iter <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// ==================================================================
</font>            <font color='#009900'>// now validate the data gradients
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num_data_inputs; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> oldval <font color='#5555FF'>=</font> subnetwork.<font color='#BB00BB'>get_output_element</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> oldval<font color='#5555FF'>*</font>base_eps;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>eps <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    eps <font color='#5555FF'>=</font> base_eps;
                subnetwork.<font color='#BB00BB'>get_output_element</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> oldval<font color='#5555FF'>+</font>eps;
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>l, subnetwork, out2<font face='Lucida Console'>)</font>;
                subnetwork.<font color='#BB00BB'>get_output_element</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> oldval<font color='#5555FF'>-</font>eps;
                impl::<font color='#BB00BB'>call_layer_forward</font><font face='Lucida Console'>(</font>l, subnetwork, out3<font face='Lucida Console'>)</font>;
                subnetwork.<font color='#BB00BB'>get_output_element</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> oldval;

                <font color='#009900'>// Compute a reference derivative via a central differences approximation and
</font>                <font color='#009900'>// compare it to the one output by the layer and make sure they match.
</font>                <font color='#0000FF'><u>double</u></font> reference_derivative <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out2,input_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out3, input_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>double</u></font> output_derivative <font color='#5555FF'>=</font> subnetwork.<font color='#BB00BB'>get_gradient_input_element</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
                output_derivative <font color='#5555FF'>-</font><font color='#5555FF'>=</font> initial_gradient_input[i];
                <font color='#0000FF'><u>double</u></font> relative_error;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>reference_derivative<font color='#5555FF'>*</font>output_derivative <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    relative_error <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>reference_derivative <font color='#5555FF'>-</font> output_derivative<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>reference_derivative<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    relative_error <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>reference_derivative <font color='#5555FF'>-</font> output_derivative<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>double</u></font> absolute_error <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>reference_derivative <font color='#5555FF'>-</font> output_derivative<font face='Lucida Console'>)</font>;
                rs_data.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>relative_error<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>relative_error<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.05</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>absolute_error<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.006</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Gradient error in data variable #</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> i <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>.  Relative error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> relative_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>expected derivative: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> reference_derivative <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>output derivative:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_derivative <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>iteration:           </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> iter <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                    <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
                <b>}</b>
            <b>}</b>

        <b>}</b> <font color='#009900'>// end for (int iter = 0; iter &lt; 5; ++iter)
</font>
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rs_params.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.003</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Average parameter gradient error is somewhat large at: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs_params.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
        <b>}</b>
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rs_data.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.003</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
            sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Average data gradient error is somewhat large at: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs_data.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
        <b>}</b>

        <font color='#0000FF'>return</font> <font color='#BB00BB'>layer_test_results</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> layer_details_type
        <font color='#5555FF'>&gt;</font>
    layer_test_results <b><a name='test_layer'></a>test_layer</b> <font face='Lucida Console'>(</font>
        layer_details_type l
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// Try a few different derivative step sizes to see if any work. 
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> base_eps <font color='#5555FF'>=</font> <font color='#979000'>0.0001</font>; base_eps <font color='#5555FF'>&lt;</font> <font color='#979000'>0.1</font>; base_eps <font color='#5555FF'>*</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>impl_test_layer</font><font face='Lucida Console'>(</font>l, base_eps<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>result<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> result;
        <b>}</b>
        <font color='#009900'>// However, if none of the step sizes worked then try this one and probably result
</font>        <font color='#009900'>// in returning an error.
</font>        <font color='#0000FF'>return</font> <font color='#BB00BB'>impl_test_layer</font><font face='Lucida Console'>(</font>l, <font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> i, <font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vlp_loop'></a>vlp_loop</b>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font color='#5555FF'>!</font>is_add_layer<font color='#5555FF'>&lt;</font>U<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type <b><a name='invoke_functor'></a>invoke_functor</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> , <font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&amp;</font> , U<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// intentionally left empty
</font>            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>is_add_layer<font color='#5555FF'>&lt;</font>U<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type <b><a name='invoke_functor'></a>invoke_functor</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> v , <font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&amp;</font> comp_i, U<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> l <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>v</font><font face='Lucida Console'>(</font>comp_i, l.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>comp_i;
            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                <font color='#0000FF'><u>size_t</u></font> comp_i,
                net_type<font color='#5555FF'>&amp;</font> net,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> v
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>invoke_functor</font><font face='Lucida Console'>(</font>v, comp_i, layer<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                vlp_loop<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>+</font><font color='#979000'>1</font>, num<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>comp_i, net,v<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vlp_loop'></a>vlp_loop</b><font color='#5555FF'>&lt;</font>num,num<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                <font color='#0000FF'><u>size_t</u></font>,
                net_type<font color='#5555FF'>&amp;</font>,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Base case of recursion.  Don't do anything.
</font>            <b>}</b>
        <b>}</b>;

    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> net_type,
        <font color='#0000FF'>typename</font> visitor
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='visit_layer_parameters'></a>visit_layer_parameters</b><font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        visitor v
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>size_t</u></font> comp_i <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        impl::vlp_loop<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>, net_type::num_layers<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>comp_i, net, v<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> i, <font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vlpg_loop'></a>vlpg_loop</b>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font><font color='#5555FF'>!</font>is_add_layer<font color='#5555FF'>&lt;</font>U<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type <b><a name='invoke_functor'></a>invoke_functor</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> , <font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&amp;</font> , U<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// intentionally left empty
</font>            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'>typename</font> std::enable_if<font color='#5555FF'>&lt;</font>is_add_layer<font color='#5555FF'>&lt;</font>U<font color='#5555FF'>&gt;</font>::value<font color='#5555FF'>&gt;</font>::type <b><a name='invoke_functor'></a>invoke_functor</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> v , <font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&amp;</font> comp_i, U<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> l <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>v</font><font face='Lucida Console'>(</font>comp_i, l.<font color='#BB00BB'>get_parameter_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>comp_i;
            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                <font color='#0000FF'><u>size_t</u></font> comp_i,
                net_type<font color='#5555FF'>&amp;</font> net,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> v
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>invoke_functor</font><font face='Lucida Console'>(</font>v, comp_i, layer<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                vlpg_loop<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>+</font><font color='#979000'>1</font>, num<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>comp_i, net,v<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vlpg_loop'></a>vlpg_loop</b><font color='#5555FF'>&lt;</font>num,num<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                <font color='#0000FF'><u>size_t</u></font>,
                net_type<font color='#5555FF'>&amp;</font>,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Base case of recursion.  Don't do anything.
</font>            <b>}</b>
        <b>}</b>;

    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> net_type,
        <font color='#0000FF'>typename</font> visitor
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='visit_layer_parameter_gradients'></a>visit_layer_parameter_gradients</b><font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        visitor v
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>size_t</u></font> comp_i <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        impl::vlpg_loop<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>, net_type::num_layers<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>comp_i, net, v<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> i, <font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vl_loop'></a>vl_loop</b>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                net_type<font color='#5555FF'>&amp;</font> net,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> v
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>v</font><font face='Lucida Console'>(</font>i, layer<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                vl_loop<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>+</font><font color='#979000'>1</font>, num<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>net,v<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vl_loop'></a>vl_loop</b><font color='#5555FF'>&lt;</font>num,num<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                net_type<font color='#5555FF'>&amp;</font>,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Base case of recursion.  Don't do anything.
</font>            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> i, <font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vl_loop_backwards'></a>vl_loop_backwards</b>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                net_type<font color='#5555FF'>&amp;</font> net,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> v
            <font face='Lucida Console'>)</font>
            <b>{</b>
                vl_loop<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>+</font><font color='#979000'>1</font>, num<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>net,v<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>v</font><font face='Lucida Console'>(</font>i, layer<font color='#5555FF'>&lt;</font>i<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font> num<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='vl_loop_backwards'></a>vl_loop_backwards</b><font color='#5555FF'>&lt;</font>num,num<font color='#5555FF'>&gt;</font>
        <b>{</b>
            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> net_type,
                <font color='#0000FF'>typename</font> visitor
                <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='visit'></a>visit</b><font face='Lucida Console'>(</font>
                net_type<font color='#5555FF'>&amp;</font>,
                visitor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Base case of recursion.  Don't do anything.
</font>            <b>}</b>
        <b>}</b>;

    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> net_type,
        <font color='#0000FF'>typename</font> visitor
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='visit_layers'></a>visit_layers</b><font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        visitor v
    <font face='Lucida Console'>)</font>
    <b>{</b>
        impl::vl_loop<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>, net_type::num_layers<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>net, v<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> net_type,
        <font color='#0000FF'>typename</font> visitor
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='visit_layers_backwards'></a>visit_layers_backwards</b><font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        visitor v
    <font face='Lucida Console'>)</font>
    <b>{</b>
        impl::vl_loop_backwards<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>, net_type::num_layers<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>net, v<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>size_t</u></font> begin,
        <font color='#0000FF'><u>size_t</u></font> end,
        <font color='#0000FF'>typename</font> net_type,
        <font color='#0000FF'>typename</font> visitor
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='visit_layers_range'></a>visit_layers_range</b><font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        visitor v
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>begin <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> end, "<font color='#CC0000'>Invalid range</font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>end <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> net_type::num_layers, "<font color='#CC0000'>Invalid range</font>"<font face='Lucida Console'>)</font>;
        impl::vl_loop<font color='#5555FF'>&lt;</font>begin,end<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>net, v<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>size_t</u></font> begin,
        <font color='#0000FF'><u>size_t</u></font> end,
        <font color='#0000FF'>typename</font> net_type,
        <font color='#0000FF'>typename</font> visitor
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='visit_layers_backwards_range'></a>visit_layers_backwards_range</b><font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        visitor v
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>begin <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> end, "<font color='#CC0000'>Invalid range</font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>end <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> net_type::num_layers, "<font color='#CC0000'>Invalid range</font>"<font face='Lucida Console'>)</font>;
        impl::vl_loop_backwards<font color='#5555FF'>&lt;</font>begin,end<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>visit</font><font face='Lucida Console'>(</font>net, v<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_DNn_CORE_H_
</font>


</pre></body></html>